package com.creditrank.spamdetector;

/*
 * The code for SpamDetector module goes in here
 * 
 * @author Sushma Reddy
 * */

import java.awt.Label;
import java.io.*;
import java.util.*;
import org.htmlparser.parserapplications.*;

public class SpamDetector {

	private FileWriter fw1;
	private  FileWriter ArffFileWriter;
	private  PrintWriter ArffPrintWriter;
	private FileReader fr1;
	private BufferedReader br1;
	private PrintWriter pw1;
	private String Class = "";
	private double AvgLengthOfWords;
	private double AvgAmountOfAnchorText;
	private double NumberOfWordsInThePage = 0;
	private double FractionofVisibleContent = 0;
	private double CompressibiltyFactor = 0;
	private int LengthOfNonMarkupWords = 0;	
	public ArrayList<Double> HeuristicValues;
	//sudhir did   public ArrayList<String> TrainedUrls;
	private static ArrayList<String > Tokens;
	private ArrayList<String > CommonWords;
	private ArrayList<String>  Ngrams;
	private static int NumberofCommonWordsinPage = 0;
	private static double FractionofPopularWords = 0;
	private static double FractionofgloballyPopularWords = 0;
	private static ArrayList<Double> Probabilities;
	private String CurrentUrl = "";
	private int UrlNumber = 0;
	
	// ******* CONSTRUCTOR FOR THE CLASS **********//
	public SpamDetector() {
		try {

		    
		    //HeuristicValues = new ArrayList<Double>();
		   // TrainedUrls = new ArrayList<String>();
		    Probabilities = new ArrayList<Double>();
		    NumberOfWordsInThePage = 0;
			FractionofVisibleContent = 0;
			CompressibiltyFactor = 0;
			LengthOfNonMarkupWords = 0;
			NumberofCommonWordsinPage = 0;
			FractionofPopularWords = 0;
			FractionofgloballyPopularWords = 0;
			
			
		} catch (Exception e) {
			e.printStackTrace();
		}
	}
	
	public void indexer(String url, ArrayList<String> CommonWords,String Label)
	{
		HeuristicValues = new ArrayList<Double>();
		//if(HeuristicValues.size() > 0) HeuristicValues.clear();
		NumberOfWordsInThePage = 0;
		FractionofVisibleContent = 0;
		CompressibiltyFactor = 0;
		LengthOfNonMarkupWords = 0;
		NumberofCommonWordsinPage = 0;
		FractionofPopularWords = 0;
		FractionofgloballyPopularWords = 0;
		Tokens = new ArrayList<String>();
		this.CommonWords = CommonWords;
		this.Class = Label;
		CurrentUrl = url;
		extractText(url);
		tokenizeText();
		 try
		 {
			  HieuristicOne();
		 }
		 catch (Exception e) 
		 {
             System.out.println(e.getMessage());
         }
			
	}
	void PrepareTrainHeaderPart()
	{
		
		if(ArffPrintWriter == null)
			ArffPrintWriter = new PrintWriter(ArffFileWriter);
		ArffPrintWriter.println("@relation ClassData");
		ArffPrintWriter.println(" ");
		
		for(int index = 1;index<10;index++)
		{
			
			ArffPrintWriter.println("@attribute "+index+" numeric");
						
		}		
		ArffPrintWriter.println("@attribute Class {normal,spam}");
		ArffPrintWriter.println(" ");
		ArffPrintWriter.println("@data");
		
		
	}
	void tokenizeText() {
        String tempStr_1, tempStr_2;
        StringTokenizer st;
        String s;
        NumberofCommonWordsinPage = 0;
        if(Tokens != null && Tokens.size() > 0) Tokens.clear();
        try {
                fr1 = new FileReader("SourceText.txt");
                br1 = new BufferedReader(fr1);
                
//                fw1 = new FileWriter("tokens.txt");                
//                pw1 = new PrintWriter(fw1);

                tempStr_1 = br1.readLine();                    
                
                while (tempStr_1 != null ) {
                    
                    tempStr_1 = tempStr_1.toLowerCase();
                    tempStr_1 = tempStr_1.trim();
                    
                    
                   tempStr_2 = tempStr_1.replaceAll("\\W", " ").replaceAll(
                            "\\d", " ");
                                    
                    st = new StringTokenizer(tempStr_2);

                    while (st.hasMoreTokens()) 
                    {
                        s = st.nextToken();
                        if(s.length()> 0)
                        {
                        	Tokens.add(s);
                        	if(this.CommonWords.contains(s.toLowerCase()))
                        		NumberofCommonWordsinPage++;
                            //pw1.println(s);
                        }
                    }
                    
                    tempStr_1 = br1.readLine();
                }

                //fw1.close();
               // pw1.close();
                
            } catch (Exception e) {
            	 System.out.println(e.getMessage());
            }

        }

    // here we extract the textual contents of the web page excluding the meta tags and html tags.
	void extractText(String url) {
		
		StringExtractor s1 = new StringExtractor(url);
		FileWriter sortVocab = null;
		PrintWriter PrintSort = null;
		try {
			
			sortVocab = new FileWriter("SourceText.txt");
			PrintSort = new PrintWriter(sortVocab);
			String contents = s1.extractStrings(false);			
			PrintSort.println(contents);
			LengthOfNonMarkupWords = contents.length();
		} catch (Exception e) { System.out.println(e.getMessage());;}
		

		PrintSort.close();
		
	}
	public void HieuristicOne()
	{
		if(Tokens.size() >0)
		{
			NumberOfWordsInThePage = Tokens.size();
			HeuristicValues.add(NumberOfWordsInThePage);
			//System.out.println("HieuristicOne Value : Number of Words in the Page : "+NumberOfWordsInThePage);
		}
		else 
			HeuristicValues.add(-1.0);
	}
	
	public void HieuristicTwo(ArrayList<String> TitleWords)
	{
		if(TitleWords.size()>0)
		{
			HeuristicValues.add(TitleWords.size()*1.0);
			//System.out.println("HieuristicTwo Value : Number of Words in the Title : "+TitleWords.size()*1.0);
		}
		else
			HeuristicValues.add(-1.0);
		
	}
	public void HieuristicThree()
	{
		if(NumberOfWordsInThePage > 0)
		{
			AvgLengthOfWords =LengthOfNonMarkupWords/NumberOfWordsInThePage;
			HeuristicValues.add(AvgLengthOfWords);
			//System.out.println("HieuristicThree Value : Average length of words: " +AvgLengthOfWords);
		}
		else
			HeuristicValues.add(-1.0);
	}
	public void HieuristicFour(ArrayList<String> AnchorWords)
	{
		if(AnchorWords.size() > 0)
		{
			AvgAmountOfAnchorText =AnchorWords.size()/NumberOfWordsInThePage;	
			HeuristicValues.add(AvgAmountOfAnchorText);			
			//System.out.println("HieuristicFour Value : Amount of Anchor Text: " +AvgAmountOfAnchorText);
		}
		else
			HeuristicValues.add(-1.0);
		AnchorWords.clear();
	}
	public void HieuristicFive(int lengthofthefile)
	{
		if(lengthofthefile> 0)
		{			
			//FractionofVisibleContent =(NumberOfWordsInThePage * 4)/(TotalWords * 4);
			FractionofVisibleContent =(LengthOfNonMarkupWords * 4.0)/(lengthofthefile * 4.0);
			HeuristicValues.add(FractionofVisibleContent);
			//System.out.println("HieuristicFive Value : Amount of Visible COntent: " +FractionofVisibleContent);
		}
		else
			HeuristicValues.add(-1.0);
	}
	public void HieuristicSix(long CompressedFileSize,long UncompressedFileSize)
	{
		//ToDo Gzip to find the COmpression Ratio.
		if(UncompressedFileSize > 0)
		{
			CompressibiltyFactor = (double)(CompressedFileSize/UncompressedFileSize);
			HeuristicValues.add(CompressibiltyFactor);
			//System.out.println("HieuristicSix Value : CompressibiltyFactor: " +CompressibiltyFactor);
		}
		else
			HeuristicValues.add(-1.0);
		
	}
	
	public void HieuristicSeven()
	{
		if(NumberOfWordsInThePage > 0.0)
		{
			FractionofPopularWords = NumberofCommonWordsinPage/NumberOfWordsInThePage;
			HeuristicValues.add(FractionofPopularWords);
			//System.out.println("HeuristicSeven Value : Fraction of Page drawn from globally popular words : "+FractionofPopularWords);
			
		}
		else
			HeuristicValues.add(-1.0);
		
	}	
	public void HieuristicEight()
	{
		//System.out.println("NumberofCommonWordsinPage : "+NumberofCommonWordsinPage);
		if(CommonWords.size() > 0)
		{
			FractionofgloballyPopularWords = NumberofCommonWordsinPage/(CommonWords.size()*1.0);
			HeuristicValues.add(FractionofgloballyPopularWords);
			//System.out.println("HeuristicEight Value : Fraction of globally popular words : "+FractionofgloballyPopularWords);			
		}
		else
			HeuristicValues.add(-1.0);
	}
	public void HieuristicNine()
	{
		
		int Numberofoccurences = 0;
		double SumofProbabilities = 0.0;
		double IndependentLikelihood = 0.0;
		if(Tokens.size() > 0)
		{
			Ngrams = new ArrayList<String>();
			for (int i=0;i<Tokens.size();i++)
			{
				if((i+1)!= Tokens.size() && (i+2)!=Tokens.size() && Tokens.get(i+1) != null&& Tokens.get(i+2) != null)
					Ngrams.add(Tokens.get(i)+" "+Tokens.get(i+1)+" "+Tokens.get(i+2));
			}
			for (int i=0;i<Ngrams.size();i++)
			{
				for (int j=0;j<Ngrams.size();j++)
				{
					if(Ngrams.get(i).matches(Ngrams.get(j)))
						Numberofoccurences+=1;
				}
				Probabilities.add(i,Math.log((Numberofoccurences * 1.0)/ (Ngrams.size() * 1.0))); 
				Numberofoccurences = 0;
			}
			
			for(int index = 0;index<Probabilities.size();index++)
			{
				SumofProbabilities += Probabilities.get(index);
			}
			if(SumofProbabilities != 0.0)
			{
				double Factor  =  1 * 1.0 /(Ngrams.size()*1.0);
				IndependentLikelihood =  - Factor * (SumofProbabilities);
				HeuristicValues.add(IndependentLikelihood);				
				//System.out.println("Independent Likelihood Factor is : "+IndependentLikelihood );
			
			}
			else
				HeuristicValues.add(-1.0);
			
			
			//PrepareArffValues(); //commented by sudhir
			Ngrams.clear();
			Probabilities.clear();
			Tokens.clear();
			ArffPrintWriter.close();
			//HeuristicValues.clear();
		}
	}
	/*private void PrepareArffValues()
	{
		//Adding the URL into an arraylist so that we can associate the predictions later.
		//System.out.println(CurrentUrl);
		//Write the attributes into the arff file.
		TrainedUrls.add(CurrentUrl);
		System.out.println("Processed Url is : "+CurrentUrl);
		for(int index = 0;index<HeuristicValues.size();index++)
		{
			
			if(index ==8)
			{
				if(HeuristicValues.get(index) != -1.0)
				{
					ArffPrintWriter.write(HeuristicValues.get(index)+","+Class);
					
				}
				else
					ArffPrintWriter.write("?,"+Class);
				
				ArffPrintWriter.write("\n");
			}
			else
			{
				if(HeuristicValues.get(index) != -1.0)
					ArffPrintWriter.write(HeuristicValues.get(index)+",");
				else
					ArffPrintWriter.write("?,");
			}	
			
		}
		
	}	

*/}


